Import and Filter TOC LEEF-2

Some preliminary setup

Code
if (!exists("params")){
   params <- rmarkdown::yaml_front_matter('~/git/0.RPackages/LEEF/measurements/LEEF.toc/inst/Import and filter TOC LEEF-2.qmd')$params
}

library(LEEF.analysis)
library(LEEF.measurement.toc)
library(dplyr)

Attaching package: 'dplyr'
The following objects are masked from 'package:stats':

    filter, lag
The following objects are masked from 'package:base':

    intersect, setdiff, setequal, union
Code
library(tidyr)
library(ggplot2)
library(ggExtra)
options(dplyr.summarise.inform = FALSE)

options(RRDdb = params$db)
Code
unlink(file.path(params$root_folder, "1.pre-processed.data", "toc"))
pre_processor_toc(file.path(params$root_folder, "0.raw.data"), file.path(params$root_folder, "1.pre-processed.data"))
[1] "Log file set to /Volumes/LEEF/0.TOC/LEEF-2//1.pre-processed.data/toc/toc.log"

########################################################

Processing toc
Warning in file.copy(file.path(input, "..", "00.general.parameter", "."), :
problem copying
/Volumes/LEEF/0.TOC/LEEF-2//0.raw.data/../00.general.parameter/. to
/Volumes/LEEF/0.TOC/LEEF-2//1.pre-processed.data/toc/.: No such file or
directory
done

########################################################
Code
unlink(file.path(params$root_folder, "2.extracted.data", "toc"))
extractor_toc(file.path(params$root_folder, "1.pre-processed.data"), file.path(params$root_folder, "2.extracted.data"))
Extracting toc
[1] "Log file set to /Volumes/LEEF/0.TOC/LEEF-2//2.extracted.data/toc/toc.log"

########################################################
Extracting toc
Processing L2_20221107Anf.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20221109Anf.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20221111A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20221114B.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20221116A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20221118A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20221121A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20221123A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20221125A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20221128A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20221130A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20221130B.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20221202A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20221205A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20221207A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20221209A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20221212A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20221214A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20221216A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20221219A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20221221A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20221223A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20221226A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20221228A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20221230A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230102A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230104A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230106A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230109A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230111A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230113A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230116A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230118A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230120A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230123A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230123B.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230125A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230127A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230130A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230130B.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230201A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230203A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230203B.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230206A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230208A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230210A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230213A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230213B.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230213C.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230215A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230217A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230220A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230222LSCB.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230224A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230224B.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230227A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230301A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230303A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230306A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230308A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230310A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230313A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230315A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230317A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230320A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230322A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230324A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230327A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230329A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230331A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230403A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230405B.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230407A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230410A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230412A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230414A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230417A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230419A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230421A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230424A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230426A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230428A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230501A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230503A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230503B!.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230505A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230505B.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230508A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230510A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230512A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230515A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230517A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230519A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230522A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230524A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230526A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230529A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230531A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230602A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230605B.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230605C.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230607A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230609A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230612A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230614A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230616A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230619A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230621A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230623A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230626A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230628A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230630A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230703A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230705A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230707A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230710A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230712A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
Processing L2_20230714A.txt ...
  |- Splitting file ...
  |- Processing metadata ...
  |- Processing parameter ...
  |- Processing calibration ...
  |- Processing actual data ...
  |- Saving files ...
done

########################################################

Read parameter and data files and determine CV and number extra measurements

Code
fns <- sapply(
  list.files(file.path(params$root_folder, "2.extracted.data", "toc"), "\\.data\\.", full.names = FALSE),
  function(fn){
    x <- strsplit(fn, "\\.")[[1]][[1]]
  }
)

data <- lapply(
    fns,
    function(fn){
      p <- read.csv(file.path(params$root_folder, "2.extracted.data", "toc", paste(fn, "parameter", "csv", sep = ".")))
      d <- read.csv(file.path(params$root_folder, "2.extracted.data", "toc", paste(fn, "data", "csv", sep = ".")))
      return(list(
        data = d, 
        parameter = p, 
        samples = max(p$samples), 
        extra_samples = max(p$extra_samples), 
        max_cv = max(p$max_cv)
      )
      )
    }
)

sapply(data, "[[", "samples") |> plot(ylab = "samples") |> print()

NULL
Code
sapply(data, "[[", "extra_samples") |> plot(ylab = "extra_samples") |> print()

NULL
Code
sapply(data, "[[", "max_cv") |> plot(ylab = "max_cv") |> print()

NULL
Code
samples <- sapply(data, "[[", "samples") |> min()

if (params$LEEF == "LEEF_1") {
  extra_samples <- sapply(data, "[[", "extra_samples") |> min()
} else if (params$LEEF == "LEEF_2") {
  extra_samples <- sapply(data, "[[", "extra_samples") |> max()
} else {
  stop("Unsopported `params$LEEF` value.", " Only `LEEF_1` and `LEEF_2` are curerently supported!")
}

max_cv <- sapply(data, "[[", "max_cv") |> max()

no_samples <- samples + extra_samples
max_col <- (2 + 3 + no_samples * 2) + 1

As the number of additional samples as well as the cv were different for some samples, we have to re-calculate conc and cv. The values which will be used are

  • samples: 2
  • extra_samples: 2
  • max_cv: 2

In case there are more extra samples, the last ones were be discarded.

Read data and discard more than 2 extra samples

Code
toc <- lapply(
  data,
  function(x){
    result <- x$data[,1:min(max_col, ncol(x$data))]
    if (ncol(x$data) < max_col) {
      sn <- paste0("conc_", 1:(extra_samples + samples))    
      snm <- sn[!(sn %in% names(result))]
      for (i in snm){
        result[,snm] <- NA
      }
    }
    return(result)
  }
) |> do.call(what = rbind)
toc$id <- 1:nrow(toc)

Remove missing inj_type

We have the following measurements files with measurements without inj_type or NULL. These can nod be analysed and need to be excluded.

Code
toc %>%
  filter(inj_type == "") %>%
  group_by(filename) %>%
  summarise(n = n()) %>% 
  collect() %>%
  knitr::kable()
filename n
L2_20230130A 4
L2_20230224A 136
L2_20230224B 8
L2_20230227A 8
L2_20230301A 8
L2_20230303A 8
L2_20230306A 8
L2_20230308A 8
L2_20230310A 8
L2_20230313A 16
L2_20230315A 16
L2_20230317A 16
L2_20230320A 16
L2_20230322A 16
L2_20230324A 16
L2_20230327A 16
L2_20230329A 16
L2_20230331A 16
L2_20230403A 16
L2_20230405B 16
L2_20230407A 16
L2_20230410A 16
L2_20230412A 16
L2_20230414A 16
L2_20230417A 16
L2_20230419A 16
L2_20230421A 16
L2_20230424A 16
L2_20230426A 11
L2_20230428A 16
L2_20230501A 16
L2_20230503A 16
L2_20230503B! 16
L2_20230505A 16
L2_20230505B 16
L2_20230508A 16
L2_20230510A 16
L2_20230512A 16
L2_20230515A 16
L2_20230517A 16
L2_20230519A 16
L2_20230522A 16
L2_20230524A 16
L2_20230526A 16
L2_20230529A 16
L2_20230531A 16
L2_20230602A 16
L2_20230605B 16
L2_20230605C 16
L2_20230607A 16
L2_20230609A 16
L2_20230612A 16
L2_20230614A 16
L2_20230616A 16
L2_20230619A 16
L2_20230621A 16
L2_20230623A 16
L2_20230626A 16
L2_20230628A 16
L2_20230630A 16
L2_20230703A 16
L2_20230705A 16
L2_20230707A 16
L2_20230710A 16
L2_20230712A 16
L2_20230714A 16

These can be filtered out

Code
before <- nrow(toc)
toc <- toc %>%
  filter(inj_type != "")
after <- nrow(toc)
cat("Before : ", before, "\n")
Before :  17100 
Code
cat("Removed: ", before - after, "\n")
Removed:  1103 
Code
cat("After  : ", after, "\n")
After  :  15997 

Remove rows from validation / null samples

In addition to the analysed samples, two more validation and null samples (H2O) were taken. Upon further consideration, these were considered as not useful and excluded from the analysis. They are saved as toc_val_samples.csv.

Code
toc %>%
  filter(identification == "H2O" | identification == "2.5mg/lIC") %>%
  group_by(filename) %>%
  summarise(n = n()) %>% 
  collect() %>%
  knitr::kable()
filename n
L2_20221107Anf 8
L2_20221109Anf 8
L2_20221111A 8
L2_20221114B 8
L2_20221116A 8
L2_20221118A 8
L2_20221121A 8
L2_20221123A 8
L2_20221125A 8
L2_20221128A 8
L2_20221130A 8
L2_20221130B 8
L2_20221202A 8
L2_20221205A 8
L2_20221207A 8
L2_20221209A 8
L2_20221212A 8
L2_20221214A 8
L2_20221216A 8
L2_20221219A 8
L2_20221221A 8
L2_20221223A 8
L2_20221226A 8
L2_20221228A 8
L2_20221230A 8
L2_20230102A 8
L2_20230104A 8
L2_20230106A 8
L2_20230109A 8
L2_20230111A 8
L2_20230113A 8
L2_20230116A 8
L2_20230118A 8
L2_20230120A 8
L2_20230123A 8
L2_20230123B 8
L2_20230125A 8
L2_20230127A 8
L2_20230130A 4
L2_20230130B 4
L2_20230201A 8
L2_20230203A 8
L2_20230203B 8
L2_20230206A 8
L2_20230208A 8
L2_20230210A 8
L2_20230213A 8
L2_20230213B 8
L2_20230213C 8
L2_20230215A 8
L2_20230217A 8
L2_20230220A 8
L2_20230222LSCB 8
L2_20230224A 4
L2_20230224B 12
L2_20230227A 16
L2_20230301A 16
L2_20230303A 16
L2_20230306A 16
L2_20230308A 16
L2_20230310A 16
L2_20230313A 24
L2_20230315A 24
L2_20230317A 24
L2_20230320A 24
L2_20230322A 24
L2_20230324A 24
L2_20230327A 24
L2_20230329A 24
L2_20230331A 24
L2_20230403A 24
L2_20230405B 24
L2_20230407A 24
L2_20230410A 24
L2_20230412A 24
L2_20230414A 24
L2_20230417A 24
L2_20230419A 24
L2_20230421A 24
L2_20230424A 24
L2_20230426A 17
L2_20230428A 24
L2_20230501A 24
L2_20230503A 24
L2_20230503B! 24
L2_20230505A 24
L2_20230505B 24
L2_20230508A 24
L2_20230510A 24
L2_20230512A 24
L2_20230515A 24
L2_20230517A 24
L2_20230519A 24
L2_20230522A 24
L2_20230524A 24
L2_20230526A 24
L2_20230529A 24
L2_20230531A 24
L2_20230602A 24
L2_20230605B 24
L2_20230605C 24
L2_20230607A 24
L2_20230609A 24
L2_20230612A 24
L2_20230614A 24
L2_20230616A 24
L2_20230619A 24
L2_20230621A 24
L2_20230623A 24
L2_20230626A 24
L2_20230628A 24
L2_20230630A 24
L2_20230703A 24
L2_20230705A 24
L2_20230707A 24
L2_20230710A 24
L2_20230712A 24
L2_20230714A 24

These can be filtered out

Code
unlink(file.path(params$root_folder, "2.extracted.data", "toc_val_samples.csv"))
toc %>%
  filter(identification == "H2O" | identification == "2.5mg/lIC") %>%
write.csv(file = file.path(params$root_folder, "2.extracted.data", "toc_val_samples.csv"))
  
before <- nrow(toc)
toc <- toc %>%
  filter(identification != "H2O" & identification != "2.5mg/lIC")
after <- nrow(toc)
cat("Before : ", before, "\n")
Before :  15997 
Code
cat("Removed: ", before - after, "\n")
Removed:  1889 
Code
cat("After  : ", after, "\n")
After  :  14108 

Check timestamp / id / Filename

In each file, there should be a maximum of 1 timestamp, except of files which contain two timestamps, i.e. which are named accodingly.

Code
toc %>%
  select(filename, timestamp, identification, inj_type) %>%
  filter(identification != "H2O" & identification != "2.5mg/lIC") %>%
  group_by(filename, timestamp) %>%
  summarise(timestamps_count = n()) %>%
  group_by(filename) %>%
  summarise(number_of_timestamps = n()) %>%
  arrange(desc(number_of_timestamps)) %>%
  knitr::kable()
filename number_of_timestamps
L2_20221107Anf 1
L2_20221109Anf 1
L2_20221111A 1
L2_20221114B 1
L2_20221116A 1
L2_20221118A 1
L2_20221121A 1
L2_20221123A 1
L2_20221125A 1
L2_20221128A 1
L2_20221130A 1
L2_20221130B 1
L2_20221202A 1
L2_20221205A 1
L2_20221207A 1
L2_20221209A 1
L2_20221212A 1
L2_20221214A 1
L2_20221216A 1
L2_20221219A 1
L2_20221221A 1
L2_20221223A 1
L2_20221226A 1
L2_20221228A 1
L2_20221230A 1
L2_20230102A 1
L2_20230104A 1
L2_20230106A 1
L2_20230109A 1
L2_20230111A 1
L2_20230113A 1
L2_20230116A 1
L2_20230118A 1
L2_20230120A 1
L2_20230123A 1
L2_20230123B 1
L2_20230125A 1
L2_20230127A 1
L2_20230130A 1
L2_20230130B 1
L2_20230201A 1
L2_20230203A 1
L2_20230203B 1
L2_20230206A 1
L2_20230208A 1
L2_20230210A 1
L2_20230213A 1
L2_20230213B 1
L2_20230213C 1
L2_20230215A 1
L2_20230217A 1
L2_20230220A 1
L2_20230222LSCB 1
L2_20230224A 1
L2_20230224B 1
L2_20230227A 1
L2_20230301A 1
L2_20230303A 1
L2_20230306A 1
L2_20230308A 1
L2_20230310A 1
L2_20230313A 1
L2_20230315A 1
L2_20230317A 1
L2_20230320A 1
L2_20230322A 1
L2_20230324A 1
L2_20230327A 1
L2_20230329A 1
L2_20230331A 1
L2_20230403A 1
L2_20230405B 1
L2_20230407A 1
L2_20230410A 1
L2_20230412A 1
L2_20230414A 1
L2_20230417A 1
L2_20230419A 1
L2_20230421A 1
L2_20230424A 1
L2_20230426A 1
L2_20230428A 1
L2_20230501A 1
L2_20230503A 1
L2_20230503B! 1
L2_20230505A 1
L2_20230505B 1
L2_20230508A 1
L2_20230510A 1
L2_20230512A 1
L2_20230515A 1
L2_20230517A 1
L2_20230519A 1
L2_20230522A 1
L2_20230524A 1
L2_20230526A 1
L2_20230529A 1
L2_20230531A 1
L2_20230602A 1
L2_20230605B 1
L2_20230605C 1
L2_20230607A 1
L2_20230609A 1
L2_20230612A 1
L2_20230614A 1
L2_20230616A 1
L2_20230619A 1
L2_20230621A 1
L2_20230623A 1
L2_20230626A 1
L2_20230628A 1
L2_20230630A 1
L2_20230703A 1
L2_20230705A 1
L2_20230707A 1
L2_20230710A 1
L2_20230712A 1
L2_20230714A 1

Fixed the following typos in the raw data. Now, it looks OK for me.

  • LEEF_21_12_10 und 21_12_13: typos in Identification
  • LEEF_22_05_18: typo in Identificatio
  • LEEF_21_11_12: File name should be LEEF_21_11_12 und 21_15_10A
  • LEEF_22_01_19 und 22_01_21A: All analysis for 22_01_21 are not valid

Set unrealistic small TC values to NA

A density plot of the values of the TC measurements in all measurements.

Code
tc <- toc %>%
  filter(inj_type == "TC")
tc <- data.frame(conc_TC = c(tc$conc_1, tc$conc_2, tc$conc_3))
tc %>% ggplot(aes(x=conc_TC)) + 
  stat_density(bw = 0.1, na.rm = TRUE) +
  geom_vline(xintercept = params$min_TC, col = "red")

One can see for TC two peaks clearly separated. The values smaller than 5 will be set to NA as they are unrealistically low and can be linked to measuring errors in the machine.

Plot the previous plot zoomed in to conc <= 10 for each bottle

conc_1

Code
toc %>%
  filter(inj_type == "TC") %>%
  ggplot(aes(x=conc_1)) + 
  geom_vline(xintercept = params$min_TC, col = "red") +
  stat_density(bw = 0.1, na.rm = TRUE) +
  facet_wrap(~bottle, ncol = 1, scales = "free_y")

conc_2

Code
toc %>%
  filter(inj_type == "TC") %>%
  ggplot(aes(x=conc_2)) + 
  geom_vline(xintercept = params$min_TC, col = "red") +
  stat_density(bw = 0.1, na.rm = TRUE) +
  facet_wrap(~bottle, ncol = 1, scales = "free_y")

conc_3

Code
toc %>%
  filter(inj_type == "TC") %>%
  ggplot(aes(x=conc_3)) + 
  geom_vline(xintercept = params$min_TC, col = "red") +
  stat_density(bw = 0.1, na.rm = TRUE) +
  facet_wrap(~bottle, ncol = 1, scales = "free_y")

Code
tc <- (toc$inj_type == "TC")
toc$conc_1[tc & sapply(toc$conc_1 < params$min_TC, isTRUE)] <- NA
toc$conc_2[tc & sapply(toc$conc_2 < params$min_TC, isTRUE)] <- NA
toc$conc_3[tc & sapply(toc$conc_3 < params$min_TC, isTRUE)] <- NA

Which are now gone

Code
tc <- toc %>%
  filter(inj_type == "TC")
tc <- data.frame(conc_TC = c(tc$conc_1, tc$conc_2, tc$conc_3))
tc %>% ggplot(aes(x=conc_TC)) + 
  stat_density(bw = 0.1, na.rm = TRUE) +
  geom_vline(xintercept = params$min_TC, col = "red")

Set unrealistic small IC values to NA

A density plot of the values of the IC measurements in all measurements.

Code
tc <- toc %>%
  filter(inj_type == "IC")
tc <- data.frame(conc_IC = c(tc$conc_1, tc$conc_2, tc$conc_3))
tc %>% 
  ggplot(aes(x=conc_IC)) + 
  geom_vline(xintercept = params$min_IC, col = "red") +
  stat_density(bw = 0.001, na.rm = TRUE)

One can see for TC two peaks clearly separated. The values smaller than 0.3 will be set to NA as they are unrealistically low and can be linked to measuring errors in the machine.

Plot the previous plot zoomed in to conc <= 3 for each bottle

conc_1

Code
toc %>%
  filter(inj_type == "IC") %>%
  ggplot(aes(x=conc_1)) + 
  geom_vline(xintercept = params$min_IC, col = "red") +
  stat_density(bw = 0.01, na.rm = TRUE) +
  facet_wrap(~bottle, ncol = 1, scales = "free_y")

conc_2

Code
toc %>%
  filter(inj_type == "IC") %>%
  ggplot(aes(x=conc_2)) + 
  geom_vline(xintercept = params$min_IC, col = "red") +
  stat_density(bw = 0.01, na.rm = TRUE) +
  facet_wrap(~bottle, ncol = 1, scales = "free_y")

conc_3

Code
toc %>%
  filter(inj_type == "IC") %>%
  ggplot(aes(x=conc_3)) + 
  geom_vline(xintercept = params$min_IC, col = "red") +
  stat_density(bw = 0.01, na.rm = TRUE) +
  facet_wrap(~bottle, ncol = 1, scales = "free_y")
Warning: Groups with fewer than two data points have been dropped.
Warning: Removed 1 rows containing missing values (`position_stack()`).

Code
tc <- (toc$inj_type == "IC")
toc$conc_1[tc & sapply(toc$conc_1 < params$min_IC, isTRUE)] <- NA
toc$conc_2[tc & sapply(toc$conc_2 < params$min_IC, isTRUE)] <- NA
toc$conc_3[tc & sapply(toc$conc_3 < params$min_IC, isTRUE)] <- NA

Which are now gone

Code
tc <- toc %>%
  filter(inj_type == "IC")
tc <- data.frame(conc_IC = c(tc$conc_1, tc$conc_2, tc$conc_3))
tc %>% ggplot(aes(x=conc_IC)) + 
  geom_vline(xintercept = params$min_IC, col = "red") +
  stat_density(bw = 0.001, na.rm = TRUE)

Recalculate conc and cv for all measurements except TOC

As the number of additional samples as well as the cv were different for some samples, we have to re-calculate conc and cv. The values which will be used are - samples: 2 - extra_samples: 2 - max_cv: 2 In case there are more extra samples, the last ones were be discarded.

As we are re-calculating TOC later, we can exclude the TOC values here.

Re-calculate conc and cv

Code
cvf <- function(x, y){
  sapply(
    1:length(x),
    function(i){
      sd(c(x[i], y[i])) / mean(c(x[i], y[i])) * 100
    }
  )
}


toc$cv_12 <- cvf(toc$conc_1, toc$conc_2)
toc$cv_13 <- cvf(toc$conc_1, toc$conc_3)
toc$cv_23 <- cvf(toc$conc_2, toc$conc_3)

toc$conc_12 <- rowMeans(toc[,c("conc_1", "conc_2")])
toc$conc_13 <- rowMeans(toc[,c("conc_1", "conc_3")])
toc$conc_23 <- rowMeans(toc[,c("conc_2", "conc_3")])

for (i in 1:nrow(toc)){
  if (toc[i,"inj_type"] == "TOC"){
    next()
  }
  if (isTRUE(as.numeric(toc[i,][["cv_12"]]) < max_cv)) {111
    toc[i,]$conc <- mean(as.numeric(toc[i,][["conc_1"]]), as.numeric(toc[i,][["conc_2"]]))
    toc[i,]$cv <- as.numeric(toc[i,][["cv_12"]])
  } else {
    sel <- suppressWarnings(
      which(
        toc[i, c("cv_12", "cv_13", "cv_23")] == min(
          as.numeric(toc[i,][["cv_12"]]), 
          as.numeric(toc[i,][["cv_13"]]), 
          as.numeric(toc[i,][["cv_23"]]), 
          na.rm = TRUE
        )
      ) 
    )
    if (length(sel) == 0) {
      toc[i,]$conc <- NA
      toc[i,]$cv <- NA
      next()
    }
    if (((as.numeric(toc[i,][c("conc_1", "conc_2", "conc_3")]) == 0) |> sum(na.rm = TRUE)) > 1) {
      toc[i,]$conc <- NA
      toc[i,]$cv <- NA
      next()
    }
    if (sel == 1) {
      toc[i,]$conc <- mean(as.numeric(toc[i,][["conc_1"]]), as.numeric(toc[i,][["conc_2"]]))
      toc[i,]$cv <- as.numeric(toc[i,][["cv_12"]])
      next()
    }
    if (sel == 2) {
      toc[i,]$conc <- mean(as.numeric(toc[i,][["conc_1"]]), as.numeric(toc[i,][["conc_3"]]))
      toc[i,]$cv <- as.numeric(toc[i,][["cv_13"]])
      next()
    }
    if (sel == 3) {
      toc[i,]$conc <- mean(as.numeric(toc[i,][["conc_2"]]), as.numeric(toc[i,][["conc_3"]]))
      toc[i,]$cv <- as.numeric(toc[i,][["cv_23"]])
      next()
    } else {
      stop("Here is something seriously wrong!!!!")
    }
  }
}

Calculate TOC

Now we re-calculate the TOC values.

Code
toc$conc[which(toc$inj_type == "TOC")] <- NA

TOC_calc <- toc %>% 
  mutate(an_id = paste(filename, position)) %>%
  pivot_wider(id_cols = an_id, names_from = inj_type, values_from = conc) %>%
  mutate(TOC_calc = TC - IC) %>%
  select(an_id, TOC_calc) %>%
  mutate(an_id = paste(an_id, "TOC"))

toc <- toc %>% 
  mutate(an_id = paste(filename, position, inj_type)) %>%
  left_join(TOC_calc, by = "an_id") %>%
  mutate( conc = ifelse(inj_type == "TOC", TOC_calc, conc) ) %>%
  select( -TOC_calc, -an_id)

Remove all where conc == NA

We have some NA values in the calculated conc values. These are removed.

Code
toc %>%
  filter(is.na(conc)) %>%
  group_by(filename) %>%
  summarise(n = n()) %>% 
  collect() %>%
  knitr::kable()
filename n
L2_20221107Anf 2
L2_20221130A 48
L2_20230123A 4
L2_20230130A 4
L2_20230203A 2
L2_20230503A 28
L2_20230505A 2
L2_20230522A 4
L2_20230605B 50
L2_20230626A 4

These can be filtered out

Code
before <- nrow(toc)
toc <- toc %>%
  filter(!is.na(conc))
after <- nrow(toc)
cat("Before : ", before, "\n")
Before :  14108 
Code
cat("Removed: ", before - after, "\n")
Removed:  148 
Code
cat("After  : ", after, "\n")
After  :  13960 

Plot after re-calculation of the TOC values

Code
toc %>%   
  ggplot(aes(x=conc)) + 
  stat_density(bw = 0.1, na.rm = TRUE) +
  facet_grid(rows = vars(inj_type), scales = "free_y")

Plot the measured concentreation per bottle of all samples

Code
toc %>%
  ggplot(aes(x=conc)) + 
  stat_density(bw = 0.1, na.rm = TRUE) +
  facet_grid(rows = vars(bottle), cols = vars(inj_type), scales = "free")

TC

Code
toc %>%
  filter(inj_type == "TC") %>%
  ggplot(aes(x=conc)) + 
  geom_vline(xintercept = 5, col = "red") +
  stat_density(bw = 0.1, na.rm = TRUE) +
  facet_wrap(~bottle, ncol = 1, scales = "free_y")

IC

Code
toc %>%
  filter(inj_type == "IC") %>%
  ggplot(aes(x=conc)) + 
  geom_vline(xintercept = 0.3, col = "red") +
  stat_density(bw = 0.01, na.rm = TRUE) +
  facet_wrap(~bottle, ncol = 1, scales = "free_y")

TOC

Code
toc %>%
  filter(inj_type == "TOC") %>%
  ggplot(aes(x=conc)) + 
  geom_vline(xintercept = 4.7, col = "red") +
  stat_density(bw = 0.1, na.rm = TRUE) +
  facet_wrap(~bottle, ncol = 1, scales = "free_y")

TN

Code
toc %>%
  filter(inj_type == "TN") %>%
  ggplot(aes(x=conc)) +
  stat_density(bw = 0.1, na.rm = TRUE) +
  facet_wrap(~bottle, ncol = 1, scales = "free_y")

Add to database

The toc is now added to the database

Code
toc_fn <- file.path(params$root_folder, "toc.rds")

options(RRDarrow = params$parquet)

saveRDS(toc, toc_fn)

dir.create(params$parquet, recursive = TRUE, showWarnings = FALSE)
parquet_add_toc(
    fn = toc_fn,
    path_to_parquet_root_dir = params$parquet,
    rename = FALSE
)

adding  TOC
Writing data...
! Be careful, path_to_parquet should be a file name, using : /Volumes/LEEF/0.TOC/LEEF-2/parquet/toc/toc.parquet
Writing data...
✔ Data are available in parquet file under /Volumes/LEEF/0.TOC/LEEF-2/parquet/toc/toc.parquet
Writing data...
Code
## Add experimenal design

object <- read.csv(file.path(params$root_folder, "experimental_design.csv"))

path_to_parquet <- file.path(params$parquet, "experimental_design", "")
dir.create(path_to_parquet, recursive = TRUE, showWarnings = FALSE)
object_to_parquet(object = object, path_to_parquet = path_to_parquet)
Writing data...
! Be careful, path_to_parquet should be a file name, using : /Volumes/LEEF/0.TOC/LEEF-2/parquet/experimental_design//experimental_design.parquet
Writing data...
✔ Data are available in parquet file under /Volumes/LEEF/0.TOC/LEEF-2/parquet/experimental_design//experimental_design.parquet
Writing data...
Code
toc_original <- toc

toc$id <- 1:nrow(toc)

Plot after filtering

Now let’s look at the plots of the measurements per bottle per timestep

Code
if (params$LEEF == "LEEF_1") {
  p1 <- plot_tocs_per_bottle_per_timestamp(db =  params$db, c("TC", "TOC"))
  p2 <- plot_tocs_per_bottle_per_timestamp(db =  params$db, c("IC"))
  p3 <- plot_tocs_per_bottle_per_timestamp(db =  params$db, c("TN"))
} else {
  p1 <- LEEF_2_plot_tocs_per_bottle_per_timestamp(db = params$parquet, c("TC", "TOC"), arrow = TRUE)
  p2 <- LEEF_2_plot_tocs_per_bottle_per_timestamp(db = params$parquet, c("IC"), arrow = TRUE)
  p3 <- LEEF_2_plot_tocs_per_bottle_per_timestamp(db = params$parquet, c("TN"), arrow = TRUE)
}
p1

Code
p2

Code
p3

Number of measurements per timestamp, bottle and inj_type

We have to look if there are any duplicate measurements.

Code
dupl <- toc %>%
  group_by(timestamp, bottle, inj_type) %>%
  filter(!is.na(bottle)) %>%
  summarise(fn_1 = min(filename), fn_2 = max(filename), count = n(), min_conc = min(conc), max_conc = max(conc)) %>%
  filter(count > 1 & count < 10) %>%  
  mutate(diff = max_conc - min_conc) %>%
  arrange(inj_type, diff)
dupl %>% 
  group_by(fn_1, fn_2, timestamp) %>%
  summarise() %>%
  arrange(timestamp) %>%
  knitr::kable()
fn_1 fn_2 timestamp
L2_20230203A L2_20230203B 20230203
L2_20230213A L2_20230213C 20230213
L2_20230503B! L2_20230505A 20230505
L2_20230503B! L2_20230505B 20230505
L2_20230605B L2_20230605C 20230605

And some plots of the duplicate concentration values only

Code
dat <- arrow_read_toc(db =  params$parquet) %>% 
  collect()
if (nrow(dat) > 0){
dat$id <- 1:nrow(dat)
ids <- dat %>% 
  filter(!is.na(bottle)) %>%
  group_by(timestamp, bottle, type) %>%
  summarize(min(id), max(id), n = n()) %>%
  filter(n > 1)
ids <- c(ids$`min(id)`, ids$`max(id)`) |>
  unique()
x <- dat %>% 
  filter(id %in% ids) %>%
  group_by(day, timestamp, bottle, type) %>%
  summarise(timestamp, bottle, type, mic = min(concentration), mac = max(concentration)) %>%
  mutate(mdiff = mac - mic)
}
Warning: There were 2 warnings in `summarise()`.
The first warning was:
ℹ In argument: `mic = min(concentration)`.
Caused by warning in `min()`:
! no non-missing arguments to min; returning Inf
ℹ Run `dplyr::last_dplyr_warnings()` to see the 1 remaining warning.
Warning: Returning more (or less) than 1 row per `summarise()` group was deprecated in
dplyr 1.1.0.
ℹ Please use `reframe()` instead.
ℹ When switching from `summarise()` to `reframe()`, remember that `reframe()`
  always returns an ungrouped data frame and adjust accordingly.

TN Duplicates

Code
if (nrow(x) > 0){
  pl <- x %>%
    filter(type == "TN") %>%
    ggplot2::ggplot(ggplot2::aes(x = mic, y = mac, colour = type)) +
    ggplot2::geom_point() +
    ggplot2::xlab("Smaler concentration Value") +
    ggplot2::ylab("Larger concentration Value")
  
  ggMarginal(pl, type="histogram")
}

TC Duplicates

Code
if (nrow(x) > 0){
  pl <- x %>%
    filter(type == "TC") %>%
    ggplot2::ggplot(ggplot2::aes(x = mic, y = mac, colour = type)) +
    ggplot2::geom_point() +
    ggplot2::xlab("Smaler concentration Value") +
    ggplot2::ylab("Larger concentration Value")
  
  ggMarginal(pl, type="histogram")
}

IC Duplicates

Code
if (nrow(x) > 0){
  pl <- x %>%
    filter(type == "IC") %>%
    ggplot2::ggplot(ggplot2::aes(x = mic, y = mac, colour = type)) +
    ggplot2::geom_point() +
    ggplot2::xlab("Smaler concentration Value") +
    ggplot2::ylab("Larger concentration Value")
  
  ggMarginal(pl, type="histogram")
}

TOC Duplicates

Code
if (nrow(x) > 0){
  pl <- x %>%
    filter(type == "TOC") %>%
    ggplot2::ggplot(ggplot2::aes(x = mic, y = mac, colour = type)) +
    ggplot2::geom_point() +
    ggplot2::xlab("Smaler concentration Value") +
    ggplot2::ylab("Larger concentration Value")
  
  ggMarginal(pl, type="histogram")
}

Final diagnostic report

Code
options(knitr.duplicate.label = "allow")
try(
  report_diagnostic(
    db = params$db,  
    template = params$LEEF,
    suffix = "TOC_added", 
    format = "html",
    lastDays = 7
  )
)